#include "stdio.h"
#include "stdlib.h"
#include "stdint.h"
#include <pthread.h>

#include "libamx.h"
#include "libavx.h"
#include "utils.h"
#include "argtable3.h"

#define __ARGTABLE

struct arg_lit  *help;
struct arg_lit  *version;
struct arg_lit  *smt;
struct arg_end  *argend;

#define AMX_WARMUP(warmup_label)      __asm__ __volatile__( \
        "movq $0x10000000, %%rax;" \
        "ldtilecfg (%0);" \
        "."warmup_label".warmup:" \
        "tdpbuud %%tmm1, %%tmm0, %%tmm2;" \
        "sub $0x1, %%rax;" \
        "jne ."warmup_label".warmup;" \
        : \
        : "r"(&tcfg_ld) \
        : "rax");

#define DIM_M   (16)
#define DIM_K   (64)
#define DIM_N   (16)

uint8_t A[DIM_M*DIM_K]         AMX_ALIGN(64) = {0};
uint8_t Borig[DIM_K*DIM_N]     AMX_ALIGN(64) = {0};
uint8_t B[(DIM_K/4)*(DIM_N*4)] AMX_ALIGN(64) = {0};
uint32_t C[DIM_M*DIM_N]        AMX_ALIGN(64) = {0};
uint32_t Ccpu[DIM_M*DIM_N]     AMX_ALIGN(64) = {0};

inline unsigned long long int rdtsc()
{
    unsigned int lo, hi;

    __asm__ volatile (".byte 0x0f, 0x31" : "=a" (lo), "=d" (hi));
    return (long long)(((unsigned long long)hi)<<32LL) | (unsigned long long) lo;
}


void matmul_dpbuud_cpu(uint32_t *c_buf, uint8_t *a_buf, uint8_t *b_buf, 
                        uint32_t M, uint32_t K, uint32_t N)
{
    int m, n, k;

    for (m = 0; m < M; m++)
        for (n = 0; n < N; n++)
            for (k = 0; k < K; k++)
            {
                c_buf[m * N + n] +=
                ((uint32_t) a_buf[m * K + k]) *
                ((uint32_t) b_buf[k * N + n]);
            }
}

void matmul_dpbssd_cpu(int32_t *c_buf, int8_t *a_buf, int8_t *b_buf, 
                        uint32_t M, uint32_t K, uint32_t N)
{
    int m, n, k;

    for (m = 0; m < M; m++)
        for (n = 0; n < N; n++)
            for (k = 0; k < K; k++)
            {
                c_buf[m * N + n] +=
                ((int32_t) a_buf[m * K + k]) *
                ((int32_t) b_buf[k * N + n]);
            }
}

uint32_t matrix_is_equal_i32(uint32_t *a_buf, uint32_t *b_buf, uint32_t dim)
{
    uint32_t flag = 1;
    for(int i=0; i<dim; ++i)
    {
        if(a_buf[i] != b_buf[i])
        {
            flag = 0;
            break;
        }
    }
    return flag;
}

void matrix_randow_gen_i8(uint8_t *pbuf, uint32_t dim, uint8_t seed)
{
    uint8_t idx = seed;
    for(int i=0; i<dim; ++i)
    {
        pbuf[i] = idx++;
    }
}

void matrix_randow_gen_i8_fix(uint8_t *pbuf, uint32_t dim, uint8_t seed)
{
    uint8_t idx = seed;
    for(int i=0; i<dim; ++i)
    {
        pbuf[i] = idx;
    }
}

static void amx_b_layout_transform(uint8_t *src, uint8_t *dst, uint32_t rows_src, uint32_t cols_src)
{
    for (int k = 0; k < rows_src/4; k++)
    {
        for (int i = 0; i < cols_src; i ++)
        {
            for (int j = 0; j < 4; j++)
            {
                *dst++ = src[j * 16 + i];
            }
        }
        src += cols_src*4;
    }
}

static double get_time(struct timespec *start,
    struct timespec *end)
{
    return end->tv_sec - start->tv_sec +
        (end->tv_nsec - start->tv_nsec) * 1e-9;
}

long long do_benchavx_inst01(int64_t loop_cnt, int64_t ops_per_loop)
{
    PROF_TMR_DECL();

    PROF_TMR_START();
    cpufp_kernel_x86_sse_fp32(loop_cnt);
    PROF_TMR_END();

    return PROF_GOPS_VALUE;
}

long long  do_benchavx_inst02(int64_t loop_cnt, int64_t ops_per_loop)
{
    PROF_TMR_DECL();

    PROF_TMR_START();
    __asm__ __volatile__(
        "movq %0, %%rax;"

        ".do_benchavx_inst02.L1:"

        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"

        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"

        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"

        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"

        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"
        "nop;nop;nop;nop;nop;nop;nop;nop;"


        "sub $0x1, %%rax;"
        "jne .do_benchavx_inst02.L1;"
        "tilerelease;"
        :
        : "r"(loop_cnt)
        : "rax");
    PROF_TMR_END();

    return PROF_GOPS_VALUE;
}

#define COLSB_0 (DIM_K)   //colsb[A]
#define COLSB_1 (DIM_N*4) //colsb[B]
#define COLSB_2 (DIM_N*sizeof(uint32_t)) //colsb[C]
#define ROWS_0  (DIM_M)   //row[A]
#define ROWS_1  (DIM_K/4) //row[B]
#define ROWS_2  (DIM_M)   //row[C]
void do_benchamx_isnt01(int64_t loop_cnt, int64_t ops_per_loop)
{
    /*
    A:16*16->colsb=16, row=16(tmm0)
    B:64*4 ->colsb=64, row=4 (tmm1)
    C:16*16->colsb=64, row=16(tmm2)
    */
    struct TileConfig tcfg_ld AMX_ALIGN(64) = 
    {
        1,
        0,
        {0},
        {COLSB_0,COLSB_1,COLSB_2,0,0,0,0,0},
        {0},
        {ROWS_0,ROWS_1,ROWS_2,0,0,0,0,0},
        {0}
    };
    
    struct TileConfig tcfg_st = {0};

    matrix_randow_gen_i8(A,     DIM_M*DIM_K, 13);//13
    matrix_randow_gen_i8(Borig, DIM_K*DIM_N, 27);//27

    DisplayIntegerMatrix(A,     DIM_M*DIM_K, "matrix_A\n",     16);
    DisplayIntegerMatrix(Borig, DIM_K*DIM_N, "matrix_Borig\n", 16);

    amx_b_layout_transform(Borig, B, DIM_K, DIM_N);
    //DisplayIntegerMatrix(B, 16*16, "matrix_B\n", 64);
    
    //dump_tilecfg_reg(&tcfg_ld, "tcfg_ld");

    amx_matmul((void*)&tcfg_ld, A, B, C);

    //amx_storecfg(&tcfg_st);
    //dump_tilecfg_reg(&tcfg_st, "tcfg_st");

    DisplayIntegerMatrix(C, DIM_M*DIM_N, "matrix_Camx\n", 16);

    matmul_dpbuud_cpu(Ccpu, A, Borig, DIM_M, DIM_K, DIM_N);
    DisplayIntegerMatrix(Ccpu, DIM_M*DIM_N, "matrix_Ccpu\n", 16);

    printf("AMX vs. CPU result is %s\n\n", matrix_is_equal_i32(C, Ccpu, DIM_M*DIM_N) ? "equal!" : "non-equal!");
}

long long  do_benchamx_inst02(int64_t loop_cnt, int64_t ops_per_loop)
{
    struct TileConfig tcfg_ld AMX_ALIGN(64) = 
    {
         1,
         0,
        {0},
        {64,64,64,64,64,64,64,64},
        {0},
        {16,16,16,16,16,16,16,16},
        {0}
    };
    int64_t stride_a = DIM_K, stride_b = DIM_N*4;
    PROF_TMR_DECL();

    /*warmup*/
    AMX_WARMUP("do_benchamx_inst02");

    PROF_TMR_START();
#if 0
    amx_matmul_benchmark(&tcfg_ld, loop_cnt);
#endif

    __asm__ __volatile__(
        "movq %5, %%rax;"
        "ldtilecfg (%0);"
        "tilezero  %%tmm0;"
        "tilezero  %%tmm1;"
        "tilezero  %%tmm2;"
        "tilezero  %%tmm3;"
        "tilezero  %%tmm4;"
        "tilezero  %%tmm5;"
        "tilezero  %%tmm6;"
        "tilezero  %%tmm7;"

        ".do_benchamx2.L1:"
        "tileloadd (%1, %3), %%tmm5;"
        "tileloadd (%2, %4), %%tmm6;"
        "tileloadd (%2, %4), %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm2;"
        "sub $0x1, %%rax;"
        "jne .do_benchamx2.L1;"


        "tilerelease;"
        :
        : "r"(&tcfg_ld), "r"(A), "r"(B), "r"(stride_a), "r"(stride_b), "r"(loop_cnt)
        : "rax");
    PROF_TMR_END();

    return PROF_GOPS_VALUE;
}

long long do_benchamx_inst03(int64_t loop_cnt, int64_t ops_per_loop)
{
    PROF_TMR_DECL();
    struct TileConfig tcfg_ld AMX_ALIGN(64) = 
    {
        1,
        0,
        {0},
        {COLSB_0,COLSB_1,COLSB_2,0,0,0,0,0},
        {0},
        {ROWS_0,ROWS_1,ROWS_2,0,0,0,0,0},
        {0}
    };
    int64_t stride_a = DIM_K, stride_b = DIM_N*4, stride_c = DIM_N*sizeof(int32_t);

    matrix_randow_gen_i8(A,     DIM_M*DIM_K, 13);//13
    matrix_randow_gen_i8(Borig, DIM_K*DIM_N, 27);//27
    amx_b_layout_transform(Borig, B, DIM_K, DIM_N);

    //arg:&cfg_ld(%0), A(%1),B(%2),C(%3),stride_a(%4),stride_b(%5),stride_c(%6)
    PROF_TMR_START();
    __asm__ __volatile__(
        "ldtilecfg (%0);"
        "tilezero  %%tmm2;"
        "tileloadd (%1, %4), %%tmm0;"
        "tileloadd (%2, %5), %%tmm1;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm2;"
        "tilestored %%tmm2, (%3, %6);"
        "tilerelease;"
        :
        : "r"(&tcfg_ld), "r"(A), "r"(B), "r"(C), "r"(stride_a), "r"(stride_b), "r"(stride_c)
        :
        );
    PROF_TMR_END();

    matmul_dpbuud_cpu(Ccpu, A, Borig, DIM_M, DIM_K, DIM_N);
    //printf("AMX vs. CPU result is %s\n\n", matrix_is_equal_i32(C, Ccpu, DIM_M*DIM_N) ? "equal!" : "non-equal!");

    return PROF_GOPS_VALUE;
}

long long  do_benchamx_inst04(int64_t loop_cnt, int64_t ops_per_loop)
{
    struct TileConfig tcfg_ld AMX_ALIGN(64) = 
    {
         1,0,{0},{64,64,64,64,64,64,64,64},{0},{16,16,16,16,16,16,16,16},{0}
    };
    int64_t stride_a = DIM_K, stride_b = DIM_N*4;
    PROF_TMR_DECL();
    /*warmup*/
    AMX_WARMUP("do_benchamx_inst04");

    PROF_TMR_START();
    __asm__ __volatile__(
        "movq %5, %%rax;"
        "ldtilecfg (%0);"
        "tilezero  %%tmm0;"
        "tilezero  %%tmm1;"
        "tilezero  %%tmm2;"
        "tilezero  %%tmm3;"
        "tilezero  %%tmm4;"
        "tilezero  %%tmm5;"
        "tilezero  %%tmm6;"
        "tilezero  %%tmm7;"

        ".do_benchamx_inst04.L1:"
        "tdpbuud %%tmm1, %%tmm0, %%tmm2;"
        "sub $0x1, %%rax;"
        "jne .do_benchamx_inst04.L1;"

        "tilerelease;"
        :
        : "r"(&tcfg_ld), "r"(A), "r"(B), "r"(stride_a), "r"(stride_b), "r"(loop_cnt)
        : "rax");
    PROF_TMR_END();

    return PROF_GOPS_VALUE;
}

long long  do_benchamx_inst05(int64_t loop_cnt, int64_t ops_per_loop)
{
    struct TileConfig tcfg_ld AMX_ALIGN(64) = 
    {
         1,0,{0},{64,64,64,64,64,64,64,64},{0},{16,16,16,16,16,16,16,16},{0}
    };
    int64_t stride_a = DIM_K, stride_b = DIM_N*4, stride_c = DIM_N*sizeof(int32_t);
    PROF_TMR_DECL();
    /*warmup*/
    AMX_WARMUP("do_benchamx_inst05");

    PROF_TMR_START();
    __asm__ __volatile__(
        "movq %7, %%rax;"
        "ldtilecfg (%0);"

        "tilezero  %%tmm0;"
        "tilezero  %%tmm1;"
        "tilezero  %%tmm2;"
        "tilezero  %%tmm3;"
        "tilezero  %%tmm4;"
        "tilezero  %%tmm5;"
        "tilezero  %%tmm6;"
        "tilezero  %%tmm7;"

        "tileloadd (%1, %4), %%tmm0;"
        "tileloadd (%1, %4), %%tmm1;"
        "tileloadd (%1, %4), %%tmm2;"
        "tileloadd (%1, %4), %%tmm3;"
        "tileloadd (%1, %4), %%tmm4;"
        "tileloadd (%1, %4), %%tmm5;"

        ".do_benchamx_inst05.L1:"

        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"
        "sqrtss %%xmm0, %%xmm0;"

        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        "tdpbuud %%tmm1, %%tmm0, %%tmm7;"
        //"tdpbuud %%tmm1, %%tmm0, %%tmm7;"

        "sub $0x1, %%rax;"
        "jne .do_benchamx_inst05.L1;"
        "tilerelease;"
        :
        : "r"(&tcfg_ld), "r"(A), "r"(B), "r"(C), "r"(stride_a), "r"(stride_b), "r"(stride_c), "r"(loop_cnt)
        : "rax");
    PROF_TMR_END();

    return PROF_GOPS_VALUE;
}

long long  do_benchamx_inst06(int64_t loop_cnt, int64_t ops_per_loop)
{
    struct TileConfig tcfg_ld AMX_ALIGN(64) = 
    {
         1,0,{0},{64,64,64,64,64,64,64,64},{0},{16,16,16,16,16,16,16,16},{0}
    };
    int64_t stride_a = DIM_K, stride_b = DIM_N*4, stride_c = DIM_N*sizeof(int32_t);
    PROF_TMR_DECL();
    /*warmup*/
    AMX_WARMUP("do_benchamx_inst06");

    PROF_TMR_START();
    __asm__ __volatile__(
        "movq %7, %%rax;"
        "ldtilecfg (%0);"
        "tilezero  %%tmm0;"
        "tilezero  %%tmm1;"
        "tilezero  %%tmm2;"
        "tilezero  %%tmm3;"
        "tilezero  %%tmm4;"
        "tilezero  %%tmm5;"
        "tilezero  %%tmm6;"
        "tilezero  %%tmm7;"

        ".do_benchamx_inst06.L1:"
        "tdpbuud %%tmm1, %%tmm0, %%tmm2;"
        "tdpbuud %%tmm3, %%tmm2, %%tmm4;"
        "sub $0x1, %%rax;"
        "jne .do_benchamx_inst06.L1;"
        "tilerelease;"
        :
        : "r"(&tcfg_ld), "r"(A), "r"(B), "r"(C), "r"(stride_a), "r"(stride_b), "r"(stride_c), "r"(loop_cnt)
        : "rax");
    PROF_TMR_END();

    return PROF_GOPS_VALUE;
}

struct benchmark_table_entry
{
    long long (*bm_func_ptr)(int64_t, int64_t);
    int64_t loop_cnt;
    int64_t ops_per_loop;
    char *name;
    char *unit;
    long long perf_result;
};

static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
void* do_benchmark(char* arg)
{
    struct benchmark_table_entry benchmark_table[] =
    {
        {do_benchavx_inst01, 100000000LL, 64LL,       "benchavx01", "tick/loop", 0},
        {do_benchavx_inst02, 100000000LL, 64LL,       "benchavx01", "tick/loop", 0},
        //{do_benchamx_inst01, 100000000LL, 64LL,       "benchamx01", "tick/loop", 0},
        //{do_benchamx_inst02, 100000000LL, 2048LL*16,  "benchamx02", "tick/loop", 0},
        //{do_benchamx_inst03, 100000000LL, 2048LL*16,  "benchamx03", "tick/loop", 0},
        {do_benchamx_inst04, 2000000LL,   2048LL*16,  "benchamx04", "tick/loop", 0},
        {do_benchamx_inst05, 2000000LL,   2048LL*16,  "benchamx05", "tick/loop", 0},
        {do_benchamx_inst06, 2000000LL,   2048LL*16,  "benchamx06", "tick/loop", 0}
    };
    double tick_percycle;

    for (int i = 0; i < sizeof(benchmark_table) / sizeof(struct benchmark_table_entry); ++i)
    {
        benchmark_table[i].perf_result = (*benchmark_table[i].bm_func_ptr)(benchmark_table[i].loop_cnt, benchmark_table[i].ops_per_loop);
    }

    if(arg != NULL) pthread_mutex_lock(&mutex);
    printf("\n==================benchmark begin==================");
    if(arg != NULL) printf("\n=========================%s=======================", arg);
    printf("\n");
    for (int i = 0; i < sizeof(benchmark_table) / sizeof(struct benchmark_table_entry); ++i)
    {
        tick_percycle = (double)benchmark_table[i].perf_result / benchmark_table[i].loop_cnt;
        //the TSC timer run at about 1.9GHz, but the core run at about 2.6GHz
        printf("%s perf = [%6.3f]gops   ~[%4lld]%s\n", benchmark_table[i].name, tick_percycle, 
                                                      (benchmark_table[i].perf_result*26)/(19*benchmark_table[i].loop_cnt), 
                                                       benchmark_table[i].unit);
    }
    printf("==================benchmark   end==================\n\n");
    if(arg != NULL) pthread_mutex_unlock(&mutex);
}

int main(int argc, char *argv[])
{
    static int smt_enable = 0;
    pthread_t thread_id_smt0;
    pthread_t thread_id_smt1;

#ifdef __ARGTABLE
    void *argtable[] = {
        help    = arg_lit0("h", "help",    "display this help and exit"),
        version = arg_lit0("v", "version", "display version info and exit"),
        smt     = arg_lit0("m", "smt",   "enable multithread version"),
        argend  = arg_end(20)
    };
    char *progname = argv[0];

    /* verify the argtable[] entries were allocated sucessfully */
    if (arg_nullcheck(argtable) != 0)
        perror("insufficient memory!");

    /* If the parser returned any errors then display them and exit */
    if (arg_parse(argc,argv,argtable) > 0)
    {
        /* Display the error details contained in the arg_end struct.*/
        arg_print_errors(stdout, argend, progname);
        perror("Try '--help' for more information!");
    }
    /* special case: '--help' takes precedence over error reporting */
    if (help->count > 0)
    {
        printf("Usage: %s", progname);
        arg_print_syntax(stdout, argtable, "\n");
        printf("List information about the FILE(s) "
               "(the current directory by default).\n\n");
        arg_print_glossary(stdout, argtable, "  %-25s %s\n");
        goto failed;
    }

    /* special case: '--version' takes precedence error reporting */
    if (version->count > 0)
    {
        printf("'%s' example program for the \"argtable\" command line argument parser.\n",progname);
        printf("03/07/2022, zhangw\n");
        goto failed;
    }

    /* special case: '--debug' takes precedence error reporting */
    if (smt->count > 0)
    {
        smt_enable = 1;
    }
#endif

    printf("[benchamx start.]\n\n");
    do_cpuid_featuredetect();
    req_xtiledata_permission();

    if(smt_enable)
    {
        pthread_create(&thread_id_smt0, NULL, (void *)*do_benchmark, "smt0");
        pthread_create(&thread_id_smt1, NULL, (void *)*do_benchmark, "smt1");
        pthread_join(thread_id_smt0, NULL);
        pthread_join(thread_id_smt1, NULL);
    }
    else
    {
        do_benchmark(0);
    }

#if 0
    do_cpuid_read(0x1d, 0x0);
    do_cpuid_read(0x1d, 0x1);
    do_cpuid_read(0x1e, 0x0);
    do_cpuid_read(0x7);
    do_cpuid_read(0x1e);
    printf("xgetbv=0x%016lx\n", xgetbv(0));
#endif

    

failed:
    /* deallocate each non-null entry in argtable[] */
    arg_freetable(argtable, sizeof(argtable) / sizeof(argtable[0]));
    printf("\n[benchmark done.]\n");
    return 0;
}
